{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7e113230-f78a-4e85-8d98-fe1ab3421b14",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import datetime as dt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1fec0d40-da53-4038-b419-e03169181381",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = '../datasets/' + 'load_ercot.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0b2493a7-9b25-4839-bd23-771f3ac38b33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "154872"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_in = pd.read_csv(filename)\n",
    "freq = 'H'\n",
    "len(df_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "492c88f3-1e15-449a-95da-f7e63b4c6113",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_in= df_in[['ds', 'y']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "45ce4e20-65fb-4074-8860-079df0b5871e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ds</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2004-01-01 01:00:00</td>\n",
       "      <td>22971.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2004-01-01 02:00:00</td>\n",
       "      <td>22191.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2004-01-01 03:00:00</td>\n",
       "      <td>21383.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2004-01-01 04:00:00</td>\n",
       "      <td>20866.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2004-01-01 05:00:00</td>\n",
       "      <td>20723.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154867</th>\n",
       "      <td>2021-08-31 20:00:00</td>\n",
       "      <td>47062.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154868</th>\n",
       "      <td>2021-08-31 21:00:00</td>\n",
       "      <td>45392.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154869</th>\n",
       "      <td>2021-08-31 22:00:00</td>\n",
       "      <td>44338.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154870</th>\n",
       "      <td>2021-08-31 23:00:00</td>\n",
       "      <td>44084.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154871</th>\n",
       "      <td>2021-09-01 00:00:00</td>\n",
       "      <td>45057.54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>154872 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         ds         y\n",
       "0       2004-01-01 01:00:00  22971.94\n",
       "1       2004-01-01 02:00:00  22191.52\n",
       "2       2004-01-01 03:00:00  21383.44\n",
       "3       2004-01-01 04:00:00  20866.56\n",
       "4       2004-01-01 05:00:00  20723.15\n",
       "...                     ...       ...\n",
       "154867  2021-08-31 20:00:00  47062.43\n",
       "154868  2021-08-31 21:00:00  45392.87\n",
       "154869  2021-08-31 22:00:00  44338.38\n",
       "154870  2021-08-31 23:00:00  44084.79\n",
       "154871  2021-09-01 00:00:00  45057.54\n",
       "\n",
       "[154872 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7b8f7369-dbaf-44fd-ad88-b555a85387f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_na = sum(df_in['y'].isnull())\n",
    "is_na"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f67f02b-4650-4008-aeb1-c047c11fd031",
   "metadata": {},
   "source": [
    "## Fill missing Dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "21246552-e981-4e2e-bd3c-f9e8a2b4dd03",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "data_len = len(df_in)\n",
    "df = df_in.copy(deep=True)\n",
    "if df[\"ds\"].dtype == np.int64:\n",
    "        df.loc[:, \"ds\"] = df.loc[:, \"ds\"].astype(str)\n",
    "df.loc[:, \"ds\"] = pd.to_datetime(df.loc[:, \"ds\"])\n",
    "\n",
    "r = pd.date_range(start=df[\"ds\"].min(), end=df[\"ds\"].max(), freq=freq)\n",
    "df = df.set_index(\"ds\").reindex(r).rename_axis(\"ds\").reset_index()\n",
    "\n",
    "num_added = len(df) - data_len\n",
    "print(num_added)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2e442aee-5a40-46b3-8cdf-b542c59b58d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_na = sum(df['y'].isnull())\n",
    "is_na"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4550872-ece5-4fa2-8444-75b3347ebc3d",
   "metadata": {
    "tags": []
   },
   "source": [
    "## fill missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e92c14a8-45f3-4134-baae-41ceb4616d6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "series = df['y']\n",
    "limit_linear = 20\n",
    "series = series.interpolate(method=\"linear\", limit=limit_linear, limit_direction=\"both\")\n",
    "\n",
    "is_na = sum(series.isnull())\n",
    "is_na"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bb7db07c-ca17-4392-b66a-8b3efcb03d7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "af5dabe4-d135-4fae-a7cf-482c2d898ca2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['y'] = series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ef7b21d7-bcbc-420b-b02e-a46983d76d78",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(filename, index=False, float_format=\"%.2f\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed34caf1-96cd-4be3-bd68-9486aa27e488",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "np-dev",
   "language": "python",
   "name": "np-dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
